using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
using Lucene.Net.Index;
using Lucene.Net.Documents;
using Lucene.Net.Analysis.Standard;

namespace LuceneIndex
{
    class Program
    {
        static void Main(string[] args)
        {
            string[] sDirectories = { "REUTE005", "REUTE006",
                "REUTE007", 
                "REUTE008", "REUTE009", "REUTE00a", "REUTE00b"
            };
            int iCount = 0;
            string INDEX_DIR = @"e:\RESEARCH\TRANSBULLETIZATION\REUTERS-INDEX-SMALL";
            IndexWriter writer = new IndexWriter(INDEX_DIR, new StandardAnalyzer(), true);
            foreach (string sD in sDirectories)
            {
                string path = @"E:\RESEARCH\TRANSBULLETIZATION\REUTERS-BULLETS\" + sD;
                
                string allTxt, DOCNO, LANG, ORGTITLE, ORGTEXT, ENTITLE, ENTEXT, SUM;
                int counter = 0;
                DirectoryInfo dir = new DirectoryInfo(path);
                FileInfo[] txtfiles = dir.GetFiles("*.*");
                Console.WriteLine("Total number of TEXT files", txtfiles.Length);
                
                
                System.Console.Out.WriteLine("Indexing to directory '" + INDEX_DIR + "'...");

                foreach (FileInfo f in txtfiles)
                {
                    if (f.FullName.EndsWith(".sum"))
                    {
                        counter++;

                        DOCNO = Regex.Replace(f.Name, "\\..*", "");

                        //////////////////////////////////////////////////  
                        //read file contnet

                        StreamReader sr = new StreamReader(f.FullName, true);
                        allTxt = sr.ReadToEnd();
                        allTxt = Regex.Replace(allTxt, "[\n\r]+", "!?@");
                        ////////////////////////////////////////////////////////////////////
                        //get LANG
                        Regex Pattern = new Regex("(<LANG>)(.*?)(</LANG>)", RegexOptions.Multiline);
                        LANG = Pattern.Match(allTxt).Groups[2].ToString().Replace("!?@", "\n");
                        if (Regex.IsMatch(LANG, "it")) LANG = "Italian";
                        else if (Regex.IsMatch(LANG, "fr")) LANG = "French";
                        else if (Regex.IsMatch(LANG, "es")) LANG = "Spanish";
                        else if (Regex.IsMatch(LANG, "ja")) LANG = "Japanese";
                        else if (Regex.IsMatch(LANG, "zh")) LANG = "Chinese";
                        DOCNO = LANG + DOCNO;
                        ////////////////////////////////////////////////////////////////////
                        //get ORGTITLE
                        Pattern = new Regex("(<ORGTITLE>)(.*?)(</ORGTITLE>)", RegexOptions.Multiline);
                        ORGTITLE = Pattern.Match(allTxt).Groups[2].ToString().Replace("!?@", "\n");
                        ////////////////////////////////////////////////////////////////////
                        //get ORGTITLE
                        Pattern = new Regex("(<ORGTEXT>)(.*?)(</ORGTEXT>)", RegexOptions.Multiline);
                        ORGTEXT = Pattern.Match(allTxt).Groups[2].ToString().Replace("!?@", "\n");
                        ////////////////////////////////////////////////////////////////////
                        //get ORGTITLE
                        Pattern = new Regex("(<ENTITLE>)(.*?)(</ENTITLE>)", RegexOptions.Multiline);
                        ENTITLE = Pattern.Match(allTxt).Groups[2].ToString().Replace("!?@", "\n");
                        ////////////////////////////////////////////////////////////////////
                        //get ORGTITLE
                        Pattern = new Regex("(<ENTEXT>*)(.*?)(</ENTEXT>)", RegexOptions.Multiline);
                        ENTEXT = Pattern.Match(allTxt).Groups[2].ToString().Replace("!?@", "\n");
                        ////////////////////////////////////////////////////////////////////
                        //get ORGTITLE
                        Pattern = new Regex("(<SUMM>)(.*?)(</SUMM>)", RegexOptions.Multiline);
                        SUM = Pattern.Match(allTxt).Groups[2].ToString().Replace("!?@", "\n");

                        //******************* Lucene ***********************
                        try
                        {
                            if (IndexDocs(writer, DOCNO, LANG, ORGTITLE, ORGTEXT, ENTITLE, ENTEXT, SUM))
                            {
                                iCount++;
                                if (iCount.ToString().EndsWith("00")) Console.Out.WriteLine(iCount);
                                if (iCount.ToString().EndsWith("000"))
                                {
                                    Console.WriteLine("Optimizing ... ");
                                    // break;
                                    writer.Optimize();
                                }
                            }
                             

                            // System.DateTime end = System.DateTime.Now;   
                            // System.Console.Out.WriteLine(end.Ticks - start.Ticks + " total milliseconds");
                        }
                        catch (System.IO.IOException e)
                        {
                            System.Console.Out.WriteLine(" caught a " + e.GetType() + "\n with message: " + e.Message);
                        }
                        // *******************************************
                        ////////////////////////////////////////////////////////////////////
                        sr.Close();
                        /////////////////////////////////////////////////////
                        ////// break;
                    }
                }
            }
            Console.Out.WriteLine("Optimizing...");
            writer.Optimize();
            writer.Close();
        }

        static Boolean IndexDocs(IndexWriter writer, string DOCNO, string LANG, string ORGTITLE, string ORGTEXT, string ENTITLE, string ENTEXT, string SUM)
        {
            Boolean output = false;
            Document newDoc= new Document() ;
            newDoc.Add(new Field("DOCNO", DOCNO, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.YES));
            
            newDoc.Add(new Field("LANG", LANG, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
            newDoc.Add(new Field("ORGTITLE", ORGTITLE, Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
            // newDoc.Add(new Field("ORGTEXT", ORGTEXT, Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
            
            newDoc.Add(new Field("ENTITLE", ENTITLE, Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
           
            //newDoc.Add(new Field("ENTEXT", ENTEXT, Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.YES));
            
            newDoc.Add(new Field("SUM", SUM, Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.YES));
            try
            {
                writer.AddDocument(newDoc);
                output = true;
            }
            catch (System.IO.FileNotFoundException ex)
            {
                System.Console.Out.WriteLine(" Error "); ex.ToString();
            }
            return output;
        }

    }


}
